1ce70c
@@ -53,6 +53,7 @@
 import org.apache.log4j.BasicConfigurator;
 import org.apache.log4j.varia.NullAppender;
 import java.lang.ClassNotFoundException;
+import org.apache.hadoop.hive.common.FileUtils;
 
 public class ExecDriver extends Task<mapredWork> implements Serializable {
 
@@ -144,10 +145,10 @@
public ExecDriver(mapredWork plan, JobConf job, boolean isSilent)
   /**
    * In Hive, when the user control-c's the command line, any running jobs
    * spawned from that command line are best-effort killed.
-   * 
+   *
    * This static constructor registers a shutdown thread to iterate over all the
    * running job kill URLs and do a get on them.
-   * 
+   *
    */
   static {
     if (new org.apache.hadoop.conf.Configuration().getBoolean(
@@ -195,7 +196,7 @@
public void jobInfo(RunningJob rj) {
           + " job  -Dmapred.job.tracker=" + hp + " -kill " + rj.getJobID());
     }
   }
-  
+
   /**
    * This class contains the state of the running task
    * Going forward, we will return this handle from execute
@@ -223,7 +224,7 @@
public Counters getCounters() throws IOException {
   }
 
   public void progress(TaskHandle taskHandle) throws IOException {
-    ExecDriverTaskHandle th = (ExecDriverTaskHandle)taskHandle; 
+    ExecDriverTaskHandle th = (ExecDriverTaskHandle)taskHandle;
     JobClient jc = th.getJobClient();
     RunningJob rj = th.getRunningJob();
     String lastReport = "";
@@ -238,7 +239,7 @@
public void progress(TaskHandle taskHandle) throws IOException {
       }
       th.setRunningJob(jc.getJob(rj.getJobID()));
       updateCounters(th);
-      
+
       String report = " map = " + this.mapProgress + "%,  reduce = " + this.reduceProgress + "%";
 
       if (!report.equals(lastReport)
@@ -286,13 +287,13 @@
public int estimateNumberOfReducers(HiveConf hive, JobConf job, mapredWork work)
     int maxReducers = hive.getIntVar(HiveConf.ConfVars.MAXREDUCERS);
     long totalInputFileSize = getTotalInputFileSize(job, work);
 
-    LOG.info("BytesPerReducer=" + bytesPerReducer + " maxReducers=" + maxReducers 
+    LOG.info("BytesPerReducer=" + bytesPerReducer + " maxReducers=" + maxReducers
         + " totalInputFileSize=" + totalInputFileSize);
 
     int reducers = (int)((totalInputFileSize + bytesPerReducer - 1) / bytesPerReducer);
     reducers = Math.max(1, reducers);
     reducers = Math.min(maxReducers, reducers);
-    return reducers;    
+    return reducers;
   }
 
   /**
@@ -301,13 +302,13 @@
public int estimateNumberOfReducers(HiveConf hive, JobConf job, mapredWork work)
   protected void setNumberOfReducers() throws IOException {
     // this is a temporary hack to fix things that are not fixed in the compiler
     Integer numReducersFromWork = work.getNumReduceTasks();
-    
+
     if(work.getReducer() == null) {
       console.printInfo("Number of reduce tasks is set to 0 since there's no reduce operator");
       work.setNumReduceTasks(Integer.valueOf(0));
     } else {
       if (numReducersFromWork >= 0) {
-        console.printInfo("Number of reduce tasks determined at compile time: " + work.getNumReduceTasks()); 
+        console.printInfo("Number of reduce tasks determined at compile time: " + work.getNumReduceTasks());
       } else if (job.getNumReduceTasks() > 0) {
         int reducers = job.getNumReduceTasks();
         work.setNumReduceTasks(reducers);
@@ -331,7 +332,7 @@
protected void setNumberOfReducers() throws IOException {
    * Calculate the total size of input files.
    * @param job the hadoop job conf.
    * @return the total size in bytes.
-   * @throws IOException 
+   * @throws IOException
    */
   public long getTotalInputFileSize(JobConf job, mapredWork work) throws IOException {
     long r = 0;
@@ -385,7 +386,7 @@
public boolean reduceDone() {
     return reduceProgress == 100;
   }
 
-  
+
   /**
    * Execute a query plan using Hadoop
    */
@@ -437,7 +438,7 @@
public int execute() {
     String auxJars = HiveConf.getVar(job, HiveConf.ConfVars.HIVEAUXJARS);
     String addedJars = HiveConf.getVar(job, HiveConf.ConfVars.HIVEADDEDJARS);
     if (StringUtils.isNotBlank(auxJars) || StringUtils.isNotBlank(addedJars)) {
-      String allJars = 
+      String allJars =
         StringUtils.isNotBlank(auxJars)
         ? (StringUtils.isNotBlank(addedJars) ? addedJars + "," + auxJars : auxJars)
         : addedJars;
@@ -455,7 +456,7 @@
public int execute() {
     if (StringUtils.isNotBlank(addedArchives)) {
       initializeFiles("tmparchives", addedArchives);
     }
-    
+
     int returnVal = 0;
     RunningJob rj = null, orig_rj = null;
     boolean success = false;
@@ -469,7 +470,7 @@
public int execute() {
       if (pwd != null)
         job.set(HiveConf.ConfVars.METASTOREPWD.varname, "HIVE");
       JobClient jc = new JobClient(job);
-      
+
 
       // make this client wait if job trcker is not behaving well.
       Throttle.checkJobTracker(job, LOG);
@@ -597,7 +598,7 @@
public static void main(String[] args) throws IOException, HiveException {
       BasicConfigurator.resetConfiguration();
       BasicConfigurator.configure(new NullAppender());
     }
-    
+
     if (planFileName == null) {
       System.err.println("Must specify Plan File Name");
       printUsage();
@@ -750,27 +751,29 @@
private boolean isEmptyPath(JobConf job, String path) throws Exception {
   /**
    * Handle a empty/null path for a given alias
    */
-  private int addInputPath(String path, JobConf job, mapredWork work, String hiveScratchDir, int numEmptyPaths, 
+  private int addInputPath(String path, JobConf job, mapredWork work, String hiveScratchDir, int numEmptyPaths,
                            boolean isEmptyPath, String alias) throws Exception {
     // either the directory does not exist or it is empty
     assert path == null || isEmptyPath;
 
     // The input file does not exist, replace it by a empty file
     Class<? extends HiveOutputFormat> outFileFormat = null;
- 
-    if (isEmptyPath) 
+
+    if (isEmptyPath)
       outFileFormat = work.getPathToPartitionInfo().get(path).getTableDesc().getOutputFileFormatClass();
     else
       outFileFormat = work.getAliasToPartnInfo().get(alias).getTableDesc().getOutputFileFormatClass();
-    
+
     // create a dummy empty file in a new directory
     String newDir = hiveScratchDir + File.separator + (++numEmptyPaths);
     Path newPath = new Path(newDir);
+    FileSystem fs = newPath.getFileSystem(job);
+    fs.mkdirs(newPath);
     String newFile = newDir + File.separator + "emptyFile";
     Path newFilePath = new Path(newFile);
 
     LOG.info("Changed input file to " + newPath.toString());
-    
+
     // toggle the work
     LinkedHashMap<String, ArrayList<String>> pathToAliases = work.getPathToAliases();
     if (isEmptyPath) {
@@ -786,7 +789,7 @@
private int addInputPath(String path, JobConf job, mapredWork work, String hiveS
     }
 
     work.setPathToAliases(pathToAliases);
-    
+
     LinkedHashMap<String,partitionDesc> pathToPartitionInfo = work.getPathToPartitionInfo();
     if (isEmptyPath) {
       pathToPartitionInfo.put(newPath.toUri().toString(), pathToPartitionInfo.get(path));
@@ -797,7 +800,7 @@
private int addInputPath(String path, JobConf job, mapredWork work, String hiveS
       pathToPartitionInfo.put(newPath.toUri().toString(), pDesc);
     }
     work.setPathToPartitionInfo(pathToPartitionInfo);
-    
+
     String onefile = newPath.toString();
     RecordWriter recWriter = outFileFormat.newInstance().getHiveRecordWriter(job, newFilePath, Text.class, false, new Properties(), null);
     recWriter.close(false);
@@ -815,13 +818,13 @@
private void addInputPaths(JobConf job, mapredWork work, String hiveScratchDir)
       LOG.info("Processing alias " + oneAlias);
       List<String> emptyPaths     = new ArrayList<String>();
 
-      // The alias may not have any path 
+      // The alias may not have any path
       String path = null;
       for (String onefile : work.getPathToAliases().keySet()) {
         List<String> aliases = work.getPathToAliases().get(onefile);
         if (aliases.contains(oneAlias)) {
           path = onefile;
-      
+
           // Multiple aliases can point to the same path - it should be processed only once
           if (pathsProcessed.contains(path))
             continue;
@@ -849,7 +852,7 @@
private void addInputPaths(JobConf job, mapredWork work, String hiveScratchDir)
         numEmptyPaths = addInputPath(null, job, work, hiveScratchDir, numEmptyPaths, false, oneAlias);
     }
   }
-  
+
   public int getType() {
     return StageType.MAPRED;
   }
